package io.github.biezhi.elves.spider;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.SocketTimeoutException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import io.github.biezhi.elves.ok.http.FastHttpClient;
import io.github.biezhi.elves.qcc.model.CompBidUrlDate;
import io.github.biezhi.elves.qcc.model.CompBidding;
import io.github.biezhi.elves.service.QccBidUrlDateService;
import io.github.biezhi.elves.utils.StringUtils;

 /**
  * 爬取企查查公告原文链接，公告时间和网页全部内容
  * @author Administrator
  *
  */
@Component
public class QccBidUrlDateSpider {
	
	@Autowired
	
	private QccBidUrlDateService qccBidUrlDateService;
	
//	private final static Executor executor = Executors.newCachedThreadPool();//启用多线程
	private final static Executor executor = Executors.newFixedThreadPool(8);//启用多线程
	
	//爬取企查查公告基本信息到数据库
	public void QccBidUrlSpiderToDB() throws Exception{
		
		//查询公告基本信息
		List<CompBidding> queryCompBiddingList = qccBidUrlDateService.queryCompBidding(120000);
		
		//关键是这一句代码，将 i 转化为  j，这样j 还是final类型的参与线程
		for(int i=0;i<queryCompBiddingList.size();i++){
				final int j=i;                                                                                                                                  
				executor.execute(new Runnable() {
		             @Override
		             public void run() {
		              try{
		            	  CompBidding compBidding = queryCompBiddingList.get(j);
		
		  				//公告id
		  				Long bidId = compBidding.getBidId();
		  				//企业id
		  				Long compId = compBidding.getCompId();
		  				//企业名称
		  				String compName = compBidding.getCompName();
		  				//公告标题
		  				String biddingTitle = compBidding.getBiddingTitle();
		  				//公告标题url
		  				String biddingTitleUrl = compBidding.getBiddingTitleUrl();
		  				
		  				//根据公告标题url爬取页面
		  				String qccBiddingHtml = spiderQccBidUrl(biddingTitleUrl);
		  				
		  				//解析获取公告发表时间，以及公告源地址
		  				Document doc = Jsoup.parse(qccBiddingHtml);
		  				String biddingDate = "";
		  				String biddingDetailUrl = doc.select("div.des > a.text-primary").get(0).attr("href");
		//  				System.out.println(biddingDate);
		  				System.out.println(Thread.currentThread().getName());
		  				System.out.println(biddingDetailUrl);
		  				
		  				//公告在企查查的基本信息
		  				CompBidUrlDate compBidUrlDate = new CompBidUrlDate();
		  				
		  				compBidUrlDate.setBiddingDate(biddingDate);
		  				compBidUrlDate.setBiddingDetailUrl(biddingDetailUrl);
		  				compBidUrlDate.setBiddingTitle(biddingTitle);
		  				compBidUrlDate.setBidId(bidId);
		  				compBidUrlDate.setCompId(compId);
		  				compBidUrlDate.setCompName(compName);
//		  				compBidUrlDate.setQccBiddingHtml(qccBiddingHtml);
		  				
		  				//插入到数据库
		  				qccBidUrlDateService.insertCompBidUrlDate(compBidUrlDate);
		  				//更新为已爬取
		  				qccBidUrlDateService.updateCompBiddingState(bidId, 1);
		
		              }catch(Exception e){
		    				//更新为已爬取
		    				qccBidUrlDateService.updateCompBiddingState(queryCompBiddingList.get(j).getBidId(), 2);
		
		              }
		             }
				});
		}
		
		System.out.println("爬取完成！");
		
	}
	
	//根据公告标题url爬取页面
	private String spiderQccBidUrl(String biddingTitleUrl){
		
		String result = "";
		try {
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			okHttpResponse = FastHttpClient
					.newBuilder()
					.build()
					.get()
					.url("https://www.qcc.com"+biddingTitleUrl)
					.build()
					.execute();
			System.out.println("https://www.qcc.com"+biddingTitleUrl);
			result = okHttpResponse.string();
			if (result.contains("<script>window.location.href='https://www.qcc.com/")) {
				System.err.println("爬取过于频繁！" + Thread.currentThread().getName());
				result = spiderQccBidUrl(biddingTitleUrl);
			}
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用spiderQccBidUrl(biddingTitleUrl)方法===");
			result = spiderQccBidUrl(biddingTitleUrl);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用spiderQccBidUrl(biddingTitleUrl)方法===");
			result = spiderQccBidUrl(biddingTitleUrl);
		}
		return result;
		
	}
	
	public String  getSubUtilSimple(String soap,String rgex){
		Pattern pattern = Pattern.compile(rgex);// 匹配的模式
		Matcher m = pattern.matcher(soap);
		String result = "";
		ArrayList<String> resultArr = new ArrayList<>();
		while (m.find()) {
			resultArr.add(m.group(1));
		}
		if (resultArr != null && resultArr.size()>0) {
			for (String str : resultArr) {
				result += str + ",";
			}
			result = result.substring(0, result.length()-1);
		}
		return result;
	}
	
	//读取文本为字符串
	public String getXmlString() {
		String xmlString;
		char[] strBuffer = null;
		int flen = 0;
		File xmlfile = new File("E:\\test.html");
		try {
		FileInputStream input = new FileInputStream(xmlfile);
		InputStreamReader in=new InputStreamReader(input,"utf-8");//字节流字符流转化的桥梁
		flen = (int)xmlfile.length();
		strBuffer = new char[flen];
		in.read(strBuffer, 0, flen);
		} catch (FileNotFoundException e) {
		e.printStackTrace();
		} catch (IOException e) {
		e.printStackTrace();
		}
		xmlString = new String(strBuffer); //构建String时，可用char[]类型，

		return xmlString;
		}
	
	public String unicodeToUtf8(String unicodeStr){
		try {
			byte[] utf8  = unicodeStr.getBytes("UTF-8");
			String utf8Str = new String(utf8,"UTF-8");
			return utf8Str;
		} catch (UnsupportedEncodingException e) {
			e.printStackTrace();
		}
		return null;
	}
	
	public static void main(String[] args) {
		QccBidUrlDateSpider qccBidUrlDateSpider = new QccBidUrlDateSpider();
		String xmlString = qccBidUrlDateSpider.getXmlString();
		String unicodeToUtf8 = qccBidUrlDateSpider.unicodeToUtf8(xmlString);
		Document doc = Jsoup.parse(unicodeToUtf8);
		String relateinfo = doc.select("div.relate-info").text();
		String rgex1 = "发布日期：(.*?)[\\s]";
		String biddingDate = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex1);
		String rgex2 = "项目编号：(.*?)[\\s]";
		String projectNo = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex2);
		String rgex3 = "地区：(.*?)[\\s]";
		String area = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex3);
		String rgex4 = "分类：(.*?)[\\s]";
		String type = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex4);
		String rgex5 = "招标/采购单位：(.*?)[\\s]";
		String purchaseComp = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex5);
		String rgex6 = "代理/委托单位：(.*?)[\\s]";
		String agent = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex6);
		String rgex7 = "开标时间：(.*?)[\\s]";
		String startBidDate = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex7);
		String rgex8 = "预算：(.*?)[\\s]";
		String budget = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex8);
		String rgex9 = "联系电话：(.*?)[\\s]";
		String tel = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex9);
		String rgex10 = "中标单位： (.*?)[\\s]";
		String winBidComp = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex10);
		String rgex11 = "中标金额：(.*?)[\\s]";
		String winBidMoney = qccBidUrlDateSpider.getSubUtilSimple(relateinfo,rgex11);
		System.out.println(relateinfo);
		System.out.println(biddingDate);
		System.out.println(projectNo);
		System.out.println(area);
		System.out.println(type);
		System.out.println(purchaseComp);
		System.out.println(agent);
		System.out.println(startBidDate);
		System.out.println(budget);
		System.out.println(tel);
		System.out.println(winBidComp);
		System.out.println(winBidMoney);
	}
}




